{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from tinyenv.flags import flags\n",
    "import numpy as np\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "\n",
    "def train():\n",
    "    mnist = input_data.read_data_sets(\n",
    "        FLAGS.data_dir, one_hot=True, fake_data=FLAGS.fake_data,\n",
    "    )\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    with tf.name_scope('input'):\n",
    "        x = tf.placeholder(tf.float32, [None, 784], name='x-input')\n",
    "        y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')\n",
    "\n",
    "    with tf.name_scope('input_reshape'):\n",
    "        image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])\n",
    "        tf.summary.image('input', image_shaped_input, 10)\n",
    "\n",
    "    def weight_variable(shape):\n",
    "        \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n",
    "        initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "        return tf.Variable(initial)\n",
    "\n",
    "    def conv2d(x, W):\n",
    "        return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding = \"SAME\" )\n",
    "      \n",
    "    def bias_variable(shape):\n",
    "        \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n",
    "        initial = tf.constant(0.1, shape=shape)\n",
    "        return tf.Variable(initial)\n",
    "\n",
    "    def variable_summaries(var):\n",
    "        with tf.name_scope('summaries'):\n",
    "            mean = tf.reduce_mean(var)\n",
    "            tf.summary.scalar('mean', mean)\n",
    "            with tf.name_scope('stddev'):\n",
    "                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))\n",
    "            tf.summary.scalar('stddev', stddev)\n",
    "            tf.summary.scalar('max', tf.reduce_max(var))\n",
    "            tf.summary.scalar('min', tf.reduce_min(var))\n",
    "            tf.summary.histogram('histogram', var)\n",
    "    \n",
    "    def max_pool(x):\n",
    "        return tf.nn.max_pool(x, ksize = [1,2,2,1], strides= [1,2,2,1], padding = \"SAME\")\n",
    "      \n",
    "    def nn_conv_pool(input_tensor,output_dim,layer_name,act=tf.nn.relu):\n",
    "        n_in = input_tensor.get_shape()[-1].value\n",
    "        with tf.name_scope(layer_name):\n",
    "          with tf.name_scope('weights'):\n",
    "              weights = weight_variable([5, 5, n_in, output_dim])\n",
    "              variable_summaries(weights)\n",
    "          with tf.name_scope('biases'):\n",
    "              biases = bias_variable([output_dim])\n",
    "              variable_summaries(biases)\n",
    "          with tf.name_scope('Wx_conv_b'):\n",
    "              preactivate = conv2d(input_tensor,weights)+biases\n",
    "              tf.summary.histogram('pre_activations',preactivate)\n",
    "          activations = act(preactivate,name='activation')\n",
    "          tf.summary.histogram('pre_activations',activations)\n",
    "          with tf.name_scope('pool'):\n",
    "              h_pool = max_pool(activations)\n",
    "          return h_pool\n",
    "    h_pool1 = nn_conv_pool(image_shaped_input, 32, 'conv_pool1')\n",
    "    h_pool2 = nn_conv_pool(h_pool1, 64, 'conv_pool2')\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1,7*7*64])         #第一个全连接层\n",
    "    \n",
    "    def nn_layer(input_tensor, input_dim, output_dim, layer_name,\n",
    "                 act=tf.nn.relu):\n",
    "        with tf.name_scope(layer_name):\n",
    "            with tf.name_scope('weights'):\n",
    "                weights = weight_variable([input_dim, output_dim])\n",
    "                variable_summaries(weights)\n",
    "            with tf.name_scope('biases'):\n",
    "                biases = bias_variable([output_dim])\n",
    "                variable_summaries(biases)\n",
    "            with tf.name_scope('Wx_plus_b'):\n",
    "                preactivate = tf.matmul(input_tensor, weights) + biases\n",
    "                tf.summary.histogram('pre_activations', preactivate)\n",
    "            activations = act(preactivate, name='activation')\n",
    "            tf.summary.histogram('activations', activations)\n",
    "            return activations\n",
    "\n",
    "    hidden1 = nn_layer(h_pool2_flat, 7*7*64, 1024, 'layer1')\n",
    "\n",
    "    with tf.name_scope('dropout'):\n",
    "        keep_prob = tf.placeholder(tf.float32)\n",
    "        tf.summary.scalar('dropout_keep_probability', keep_prob)\n",
    "        dropped = tf.nn.dropout(hidden1, keep_prob)\n",
    "\n",
    "    y = nn_layer(dropped, 1024, 10, 'layer2', act=tf.identity)\n",
    "\n",
    "    with tf.name_scope('cross_entropy'):\n",
    "        diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "        with tf.name_scope('total'):\n",
    "            cross_entropy = tf.reduce_mean(diff)\n",
    "    tf.summary.scalar('cross_entropy', cross_entropy)\n",
    "\n",
    "    with tf.name_scope('train'):\n",
    "        train_step = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(\n",
    "            cross_entropy)\n",
    "\n",
    "    with tf.name_scope('accuracy'):\n",
    "        with tf.name_scope('correct_prediction'):\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        with tf.name_scope('accuracy'):\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "\n",
    "    merged = tf.summary.merge_all()\n",
    "    train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)\n",
    "    test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test')\n",
    "    tf.global_variables_initializer().run()\n",
    "\n",
    "    def feed_dict(train):\n",
    "        if train or FLAGS.fake_data:\n",
    "            xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data)\n",
    "            k = FLAGS.dropout\n",
    "        else:\n",
    "            xs, ys = mnist.test.images, mnist.test.labels\n",
    "            k = 1.0\n",
    "        return {x: xs, y_: ys, keep_prob: k}\n",
    "\n",
    "    for i in range(FLAGS.iterations):\n",
    "        if i % 10 == 0:  # Record summaries and test-set accuracy\n",
    "            summary, acc = sess.run(\n",
    "                [merged, accuracy], feed_dict=feed_dict(False))\n",
    "            test_writer.add_summary(summary, i)\n",
    "            print('Accuracy at step %s: %s' % (i, acc))\n",
    "        else:\n",
    "            if i % 100 == 99:\n",
    "                run_options = tf.RunOptions(\n",
    "                    trace_level=tf.RunOptions.FULL_TRACE)\n",
    "                run_metadata = tf.RunMetadata()\n",
    "                summary, _ = sess.run([merged, train_step],\n",
    "                                      feed_dict=feed_dict(True),\n",
    "                                      options=run_options,\n",
    "                                      run_metadata=run_metadata)\n",
    "                train_writer.add_run_metadata(run_metadata, 'step%03d' % i)\n",
    "                train_writer.add_summary(summary, i)\n",
    "            else:\n",
    "                summary, _ = sess.run(\n",
    "                    [merged, train_step], feed_dict=feed_dict(True))\n",
    "                train_writer.add_summary(summary, i)\n",
    "    train_writer.close()\n",
    "    test_writer.close()\n",
    "\n",
    "\n",
    "def main(_):\n",
    "    if tf.gfile.Exists(FLAGS.log_dir):\n",
    "        tf.gfile.DeleteRecursively(FLAGS.log_dir)\n",
    "    tf.gfile.MakeDirs(FLAGS.log_dir)\n",
    "    train()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    FLAGS = flags()\n",
    "    tf.app.run(main=main, argv=[sys.argv[0]])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
